{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: Custom nn Modules\n",
    "--------------------------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer, trained to predict y from x\n",
    "by minimizing squared Euclidean distance.\n",
    "\n",
    "<strong style=\"color:red\">This implementation defines the model as a custom Module subclass.</strong>\n",
    "Whenever you want a model more complex than a simple sequence of existing Modules you will need to define your model this way.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 709.3311767578125\n",
      "1 654.7893676757812\n",
      "2 608.1807250976562\n",
      "3 567.7784423828125\n",
      "4 531.8323364257812\n",
      "5 499.66741943359375\n",
      "6 470.4566345214844\n",
      "7 443.8223571777344\n",
      "8 419.05169677734375\n",
      "9 396.0458679199219\n",
      "10 374.43109130859375\n",
      "11 354.20361328125\n",
      "12 335.1230773925781\n",
      "13 317.0469970703125\n",
      "14 299.84332275390625\n",
      "15 283.5288391113281\n",
      "16 268.0382995605469\n",
      "17 253.37509155273438\n",
      "18 239.4454345703125\n",
      "19 226.2010498046875\n",
      "20 213.5829620361328\n",
      "21 201.6224365234375\n",
      "22 190.28240966796875\n",
      "23 179.51385498046875\n",
      "24 169.25572204589844\n",
      "25 159.55300903320312\n",
      "26 150.33572387695312\n",
      "27 141.61219787597656\n",
      "28 133.38975524902344\n",
      "29 125.64966583251953\n",
      "30 118.36223602294922\n",
      "31 111.47442626953125\n",
      "32 104.9749984741211\n",
      "33 98.85650634765625\n",
      "34 93.07905578613281\n",
      "35 87.63988494873047\n",
      "36 82.53087615966797\n",
      "37 77.69802856445312\n",
      "38 73.15835571289062\n",
      "39 68.889404296875\n",
      "40 64.88677215576172\n",
      "41 61.11894989013672\n",
      "42 57.58547592163086\n",
      "43 54.261871337890625\n",
      "44 51.14573669433594\n",
      "45 48.219093322753906\n",
      "46 45.471214294433594\n",
      "47 42.88657760620117\n",
      "48 40.45998001098633\n",
      "49 38.17283248901367\n",
      "50 36.02518081665039\n",
      "51 34.00325393676758\n",
      "52 32.10536575317383\n",
      "53 30.320703506469727\n",
      "54 28.644628524780273\n",
      "55 27.06778335571289\n",
      "56 25.58408546447754\n",
      "57 24.185808181762695\n",
      "58 22.868019104003906\n",
      "59 21.6263370513916\n",
      "60 20.4561824798584\n",
      "61 19.346637725830078\n",
      "62 18.30196189880371\n",
      "63 17.318538665771484\n",
      "64 16.393041610717773\n",
      "65 15.520407676696777\n",
      "66 14.696928977966309\n",
      "67 13.919346809387207\n",
      "68 13.18565845489502\n",
      "69 12.491520881652832\n",
      "70 11.835188865661621\n",
      "71 11.2154541015625\n",
      "72 10.630057334899902\n",
      "73 10.077787399291992\n",
      "74 9.556388854980469\n",
      "75 9.062957763671875\n",
      "76 8.596435546875\n",
      "77 8.155970573425293\n",
      "78 7.7393317222595215\n",
      "79 7.345122814178467\n",
      "80 6.972507476806641\n",
      "81 6.619952201843262\n",
      "82 6.286576271057129\n",
      "83 5.970588207244873\n",
      "84 5.671485424041748\n",
      "85 5.388254642486572\n",
      "86 5.120092868804932\n",
      "87 4.8664093017578125\n",
      "88 4.626236438751221\n",
      "89 4.398593902587891\n",
      "90 4.183012008666992\n",
      "91 3.9791600704193115\n",
      "92 3.785944700241089\n",
      "93 3.60274600982666\n",
      "94 3.428943395614624\n",
      "95 3.2638845443725586\n",
      "96 3.1075387001037598\n",
      "97 2.959308624267578\n",
      "98 2.8187263011932373\n",
      "99 2.6853179931640625\n",
      "100 2.558555841445923\n",
      "101 2.438171625137329\n",
      "102 2.3240578174591064\n",
      "103 2.2159361839294434\n",
      "104 2.1129648685455322\n",
      "105 2.0151138305664062\n",
      "106 1.9221254587173462\n",
      "107 1.8335055112838745\n",
      "108 1.7493976354599\n",
      "109 1.6694045066833496\n",
      "110 1.5932769775390625\n",
      "111 1.5209017992019653\n",
      "112 1.4520049095153809\n",
      "113 1.3862475156784058\n",
      "114 1.3237149715423584\n",
      "115 1.2642605304718018\n",
      "116 1.207702875137329\n",
      "117 1.1538304090499878\n",
      "118 1.1025103330612183\n",
      "119 1.0536097288131714\n",
      "120 1.0070186853408813\n",
      "121 0.9626676440238953\n",
      "122 0.920424222946167\n",
      "123 0.8801770806312561\n",
      "124 0.8418165445327759\n",
      "125 0.805270254611969\n",
      "126 0.7704170942306519\n",
      "127 0.7371296286582947\n",
      "128 0.7054294943809509\n",
      "129 0.6751862168312073\n",
      "130 0.6463737487792969\n",
      "131 0.6188382506370544\n",
      "132 0.5925520658493042\n",
      "133 0.5674505829811096\n",
      "134 0.5434970855712891\n",
      "135 0.5206201672554016\n",
      "136 0.49875980615615845\n",
      "137 0.47791364789009094\n",
      "138 0.4580010771751404\n",
      "139 0.43897461891174316\n",
      "140 0.42081305384635925\n",
      "141 0.4034288823604584\n",
      "142 0.38681507110595703\n",
      "143 0.37091976404190063\n",
      "144 0.35573336482048035\n",
      "145 0.3412199318408966\n",
      "146 0.3273349702358246\n",
      "147 0.3140583634376526\n",
      "148 0.3013553321361542\n",
      "149 0.28920453786849976\n",
      "150 0.2775731086730957\n",
      "151 0.26644518971443176\n",
      "152 0.2558402121067047\n",
      "153 0.24569058418273926\n",
      "154 0.23595935106277466\n",
      "155 0.2266429364681244\n",
      "156 0.2177184671163559\n",
      "157 0.20916777849197388\n",
      "158 0.20099204778671265\n",
      "159 0.19316202402114868\n",
      "160 0.18565788865089417\n",
      "161 0.1784757524728775\n",
      "162 0.17158988118171692\n",
      "163 0.16498415172100067\n",
      "164 0.15864711999893188\n",
      "165 0.15257051587104797\n",
      "166 0.146744042634964\n",
      "167 0.14115436375141144\n",
      "168 0.13580025732517242\n",
      "169 0.13065996766090393\n",
      "170 0.1257256418466568\n",
      "171 0.12099218368530273\n",
      "172 0.11645203828811646\n",
      "173 0.11209198832511902\n",
      "174 0.10791150480508804\n",
      "175 0.10389424860477448\n",
      "176 0.10003500431776047\n",
      "177 0.09633102267980576\n",
      "178 0.09277655184268951\n",
      "179 0.0893627405166626\n",
      "180 0.08608417212963104\n",
      "181 0.08293433487415314\n",
      "182 0.07990642637014389\n",
      "183 0.07699478417634964\n",
      "184 0.07419616729021072\n",
      "185 0.07150783389806747\n",
      "186 0.06892382353544235\n",
      "187 0.06643807888031006\n",
      "188 0.06404571235179901\n",
      "189 0.061745885759592056\n",
      "190 0.05953596532344818\n",
      "191 0.057411134243011475\n",
      "192 0.055364083498716354\n",
      "193 0.05339554697275162\n",
      "194 0.05150125175714493\n",
      "195 0.04967724159359932\n",
      "196 0.04792153090238571\n",
      "197 0.0462322011590004\n",
      "198 0.04460521042346954\n",
      "199 0.043040405958890915\n",
      "200 0.04153428226709366\n",
      "201 0.04008162021636963\n",
      "202 0.03868435323238373\n",
      "203 0.03733862191438675\n",
      "204 0.036042314022779465\n",
      "205 0.03479497879743576\n",
      "206 0.033591385930776596\n",
      "207 0.03243234008550644\n",
      "208 0.03131501004099846\n",
      "209 0.03023836947977543\n",
      "210 0.02920081652700901\n",
      "211 0.02820109948515892\n",
      "212 0.027237920090556145\n",
      "213 0.026308931410312653\n",
      "214 0.025412991642951965\n",
      "215 0.024548782035708427\n",
      "216 0.0237163957208395\n",
      "217 0.02291492559015751\n",
      "218 0.022140245884656906\n",
      "219 0.02139277756214142\n",
      "220 0.020672772079706192\n",
      "221 0.019977817311882973\n",
      "222 0.01930769719183445\n",
      "223 0.018660927191376686\n",
      "224 0.01803659088909626\n",
      "225 0.01743418164551258\n",
      "226 0.01685318350791931\n",
      "227 0.01629215106368065\n",
      "228 0.015750277787446976\n",
      "229 0.015227959491312504\n",
      "230 0.01472394447773695\n",
      "231 0.014237134717404842\n",
      "232 0.013767163269221783\n",
      "233 0.013313953764736652\n",
      "234 0.012876011431217194\n",
      "235 0.012452812865376472\n",
      "236 0.01204437855631113\n",
      "237 0.011649489402770996\n",
      "238 0.011268414556980133\n",
      "239 0.010900527238845825\n",
      "240 0.010544807650148869\n",
      "241 0.010201451368629932\n",
      "242 0.009869597852230072\n",
      "243 0.009548957459628582\n",
      "244 0.00923930387943983\n",
      "245 0.008940378203988075\n",
      "246 0.008651313371956348\n",
      "247 0.00837192963808775\n",
      "248 0.008101791143417358\n",
      "249 0.007840652018785477\n",
      "250 0.007588385604321957\n",
      "251 0.00734443124383688\n",
      "252 0.007108722813427448\n",
      "253 0.006880840752273798\n",
      "254 0.006660473998636007\n",
      "255 0.006447690539062023\n",
      "256 0.006242049392312765\n",
      "257 0.006042822729796171\n",
      "258 0.005850312765687704\n",
      "259 0.005664472933858633\n",
      "260 0.005484485533088446\n",
      "261 0.005310356616973877\n",
      "262 0.005142012611031532\n",
      "263 0.004978943150490522\n",
      "264 0.004821403883397579\n",
      "265 0.0046689449809491634\n",
      "266 0.004521530587226152\n",
      "267 0.004378848243504763\n",
      "268 0.004240745212882757\n",
      "269 0.004107245709747076\n",
      "270 0.003978163469582796\n",
      "271 0.0038531292229890823\n",
      "272 0.003732184646651149\n",
      "273 0.0036151257809251547\n",
      "274 0.0035018951166421175\n",
      "275 0.0033924004528671503\n",
      "276 0.003286367980763316\n",
      "277 0.003183819353580475\n",
      "278 0.003084411146119237\n",
      "279 0.002988280262798071\n",
      "280 0.002895267214626074\n",
      "281 0.0028051119297742844\n",
      "282 0.002717931056395173\n",
      "283 0.0026335057336837053\n",
      "284 0.0025517733301967382\n",
      "285 0.0024726183619350195\n",
      "286 0.0023959518875926733\n",
      "287 0.0023217727430164814\n",
      "288 0.002249920042231679\n",
      "289 0.0021804841235280037\n",
      "290 0.0021131583489477634\n",
      "291 0.0020479350350797176\n",
      "292 0.0019847778603434563\n",
      "293 0.0019237393280491233\n",
      "294 0.0018645116360858083\n",
      "295 0.001807193853892386\n",
      "296 0.001751688658259809\n",
      "297 0.0016978615894913673\n",
      "298 0.001645818818360567\n",
      "299 0.0015953757101669908\n",
      "300 0.0015464427415281534\n",
      "301 0.0014990997733548284\n",
      "302 0.0014532479690387845\n",
      "303 0.0014088189927861094\n",
      "304 0.0013657839735969901\n",
      "305 0.0013240539701655507\n",
      "306 0.0012836408568546176\n",
      "307 0.0012445047032088041\n",
      "308 0.0012066259514540434\n",
      "309 0.0011698707239702344\n",
      "310 0.0011342811631038785\n",
      "311 0.0010998051147907972\n",
      "312 0.0010663592256605625\n",
      "313 0.0010339710861444473\n",
      "314 0.0010025850497186184\n",
      "315 0.0009721811511553824\n",
      "316 0.0009426860488019884\n",
      "317 0.0009141486370936036\n",
      "318 0.0008864619885571301\n",
      "319 0.0008596628322266042\n",
      "320 0.0008336720056831837\n",
      "321 0.0008084867149591446\n",
      "322 0.0007840580074116588\n",
      "323 0.0007603677804581821\n",
      "324 0.0007374538690783083\n",
      "325 0.0007152230245992541\n",
      "326 0.0006936740828678012\n",
      "327 0.0006727813160978258\n",
      "328 0.000652538612484932\n",
      "329 0.0006328915478661656\n",
      "330 0.0006138565368019044\n",
      "331 0.0005954185035079718\n",
      "332 0.0005775362369604409\n",
      "333 0.0005601969314739108\n",
      "334 0.0005433841142803431\n",
      "335 0.0005270906258374453\n",
      "336 0.00051127775805071\n",
      "337 0.0004959790967404842\n",
      "338 0.0004811157996300608\n",
      "339 0.0004667129251174629\n",
      "340 0.00045276738819666207\n",
      "341 0.0004392324772197753\n",
      "342 0.00042611922253854573\n",
      "343 0.0004134217160753906\n",
      "344 0.000401092431275174\n",
      "345 0.00038916213088668883\n",
      "346 0.00037757866084575653\n",
      "347 0.00036635002470575273\n",
      "348 0.0003554584109224379\n",
      "349 0.00034489837707951665\n",
      "350 0.0003346536250319332\n",
      "351 0.00032472601742483675\n",
      "352 0.0003150880802422762\n",
      "353 0.0003057374560739845\n",
      "354 0.00029666267801076174\n",
      "355 0.0002878709929063916\n",
      "356 0.0002793623716570437\n",
      "357 0.00027109854272566736\n",
      "358 0.00026306984364055097\n",
      "359 0.00025529274716973305\n",
      "360 0.0002477477246429771\n",
      "361 0.00024043054145295173\n",
      "362 0.00023332696582656354\n",
      "363 0.00022643627016805112\n",
      "364 0.00021975069830659777\n",
      "365 0.0002132808294845745\n",
      "366 0.00020698511798400432\n",
      "367 0.00020088373275939375\n",
      "368 0.00019496219465509057\n",
      "369 0.00018922310846392065\n",
      "370 0.00018364931747782975\n",
      "371 0.00017825528630055487\n",
      "372 0.0001729984796838835\n",
      "373 0.00016792103997431695\n",
      "374 0.00016297858383040875\n",
      "375 0.0001581926189828664\n",
      "376 0.00015354689094237983\n",
      "377 0.00014903568080626428\n",
      "378 0.00014465997810475528\n",
      "379 0.0001404182257829234\n",
      "380 0.0001362980401609093\n",
      "381 0.0001323022588621825\n",
      "382 0.0001284277968807146\n",
      "383 0.00012465678446460515\n",
      "384 0.00012101092579541728\n",
      "385 0.00011746882955776528\n",
      "386 0.00011402999371057376\n",
      "387 0.00011069478205172345\n",
      "388 0.00010745646432042122\n",
      "389 0.00010431263945065439\n",
      "390 0.00010126323468284681\n",
      "391 9.83058926067315e-05\n",
      "392 9.542918269289657e-05\n",
      "393 9.264467371394858e-05\n",
      "394 8.993973460746929e-05\n",
      "395 8.731595880817622e-05\n",
      "396 8.476770744891837e-05\n",
      "397 8.229471859522164e-05\n",
      "398 7.989227742655203e-05\n",
      "399 7.756721606710926e-05\n",
      "400 7.530670700361952e-05\n",
      "401 7.311245281016454e-05\n",
      "402 7.098258356563747e-05\n",
      "403 6.891821976751089e-05\n",
      "404 6.691137969028205e-05\n",
      "405 6.496450077975169e-05\n",
      "406 6.307119474513456e-05\n",
      "407 6.123555067460984e-05\n",
      "408 5.945600787526928e-05\n",
      "409 5.77285754843615e-05\n",
      "410 5.6049448176054284e-05\n",
      "411 5.4421936511062086e-05\n",
      "412 5.284184589982033e-05\n",
      "413 5.130580029799603e-05\n",
      "414 4.981768142897636e-05\n",
      "415 4.837166488869116e-05\n",
      "416 4.696755422628485e-05\n",
      "417 4.5605724153574556e-05\n",
      "418 4.42837699665688e-05\n",
      "419 4.299794090911746e-05\n",
      "420 4.175202775513753e-05\n",
      "421 4.054423334309831e-05\n",
      "422 3.936677967431024e-05\n",
      "423 3.82317193725612e-05\n",
      "424 3.7120327760931104e-05\n",
      "425 3.6045377783011645e-05\n",
      "426 3.500339516904205e-05\n",
      "427 3.3988482755376026e-05\n",
      "428 3.300622120150365e-05\n",
      "429 3.204854147043079e-05\n",
      "430 3.112287959083915e-05\n",
      "431 3.022178316314239e-05\n",
      "432 2.934937583631836e-05\n",
      "433 2.850165947165806e-05\n",
      "434 2.7679227059707046e-05\n",
      "435 2.6879926736000925e-05\n",
      "436 2.6102839910890907e-05\n",
      "437 2.534805935283657e-05\n",
      "438 2.4618644602014683e-05\n",
      "439 2.3905065972940065e-05\n",
      "440 2.3216734916786663e-05\n",
      "441 2.2543872546521015e-05\n",
      "442 2.1896390535403043e-05\n",
      "443 2.12654413189739e-05\n",
      "444 2.065249645966105e-05\n",
      "445 2.0055260392837226e-05\n",
      "446 1.9479120965115726e-05\n",
      "447 1.8916914996225387e-05\n",
      "448 1.8371170881437138e-05\n",
      "449 1.7842041415860876e-05\n",
      "450 1.7325937733403407e-05\n",
      "451 1.6828329535201192e-05\n",
      "452 1.634438922337722e-05\n",
      "453 1.5873265510890633e-05\n",
      "454 1.5416211681440473e-05\n",
      "455 1.4974109035392758e-05\n",
      "456 1.454072116757743e-05\n",
      "457 1.412294659530744e-05\n",
      "458 1.3716983630729374e-05\n",
      "459 1.3322853192221373e-05\n",
      "460 1.2940706255903933e-05\n",
      "461 1.256680116057396e-05\n",
      "462 1.220697959070094e-05\n",
      "463 1.1855883712996729e-05\n",
      "464 1.151378182839835e-05\n",
      "465 1.1184628419869114e-05\n",
      "466 1.086298652808182e-05\n",
      "467 1.055144275596831e-05\n",
      "468 1.0248703802062664e-05\n",
      "469 9.953941116691567e-06\n",
      "470 9.66835068538785e-06\n",
      "471 9.391327694174834e-06\n",
      "472 9.120443792198785e-06\n",
      "473 8.858366527420003e-06\n",
      "474 8.605114089732524e-06\n",
      "475 8.358725608559325e-06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "476 8.11725840321742e-06\n",
      "477 7.885952072683722e-06\n",
      "478 7.65842469263589e-06\n",
      "479 7.440140507242177e-06\n",
      "480 7.225977242342196e-06\n",
      "481 7.0188575591600966e-06\n",
      "482 6.818445399403572e-06\n",
      "483 6.622934051847551e-06\n",
      "484 6.4338132688135374e-06\n",
      "485 6.249259513424477e-06\n",
      "486 6.06979165240773e-06\n",
      "487 5.896843504160643e-06\n",
      "488 5.727140433009481e-06\n",
      "489 5.5636855904594995e-06\n",
      "490 5.403326667874353e-06\n",
      "491 5.249598871159833e-06\n",
      "492 5.09859864905593e-06\n",
      "493 4.952773451805115e-06\n",
      "494 4.8109823183040135e-06\n",
      "495 4.673816874856129e-06\n",
      "496 4.540507688943762e-06\n",
      "497 4.4102980609750375e-06\n",
      "498 4.283742782718036e-06\n",
      "499 4.1610874177422374e-06\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        In the constructor we instantiate two nn.Linear modules and assign them as\n",
    "        member variables.\n",
    "        \"\"\"\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        In the forward function we accept a Variable of input data and we must return\n",
    "        a Variable of output data. We can use Modules defined in the constructor as\n",
    "        well as arbitrary operators on Variables.\n",
    "        \"\"\"\n",
    "        h_relu = self.linear1(x).clamp(min=0)\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred\n",
    "\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold inputs and outputs, and wrap them in Variables\n",
    "x = Variable(torch.randn(N, D_in))\n",
    "y = Variable(torch.randn(N, D_out), requires_grad=False)\n",
    "\n",
    "# Construct our model by instantiating the class defined above\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "\n",
    "# Construct our loss function and an Optimizer. The call to model.parameters()\n",
    "# in the SGD constructor will contain the learnable parameters of the two\n",
    "# nn.Linear modules which are members of the model.\n",
    "criterion = torch.nn.MSELoss(size_average=False)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)\n",
    "for t in range(500):\n",
    "    # Forward pass: Compute predicted y by passing x to the model\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Zero gradients, perform a backward pass, and update the weights.\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
